{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "from IPython.core.interactiveshell import InteractiveShell\n",
    "InteractiveShell.ast_node_interactivity = 'all'  # default is ‘last_expr'\n",
    "\n",
    "%load_ext autoreload\n",
    "%autoreload 2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "from shutil import copy\n",
    "\n",
    "import azureml.core\n",
    "from azureml.core import (Workspace, Experiment, Datastore, Dataset, \n",
    "                          ContainerRegistry, ScriptRunConfig, RunConfiguration, \n",
    "                          Run)\n",
    "from azureml.data.datapath import DataPath\n",
    "from azureml.data.data_reference import DataReference\n",
    "from azureml.core.runconfig import DataReferenceConfiguration\n",
    "from azureml.tensorboard import Tensorboard"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Version of AML: 1.0.83\n"
     ]
    }
   ],
   "source": [
    "print('Version of AML: {}'.format(azureml.core.__version__))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# MegaDetector v4 training\n",
    "\n",
    "https://docs.microsoft.com/en-us/azure/machine-learning/how-to-set-up-training-targets#amlcompute\n",
    "    "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Provide credentials\n",
    "\n",
    "Provide the account name and the key to the storage account, and password to the container registry where the base image is."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "storage_account_name = os.environ.get('STORAGE_ACCOUNT_NAME')\n",
    "storage_account_key = os.environ.get('STORAGE_ACCOUNT_KEY')\n",
    "registry_pw = os.environ.get('REGISTRY_PASSWORD')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Connect to the AML workspace"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "WARNING - Note, we have launched a browser for you to login. For old experience with device code, use \"az login --use-device-code\"\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Performing interactive authentication. Please follow the instructions on the terminal.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "WARNING - You have logged in. Now let us find all the subscriptions to which you have access...\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Interactive authentication successfully completed.\n",
      "siyu_sc\tsouthcentralus\tyasiyu_rg\tsouthcentralus\n"
     ]
    }
   ],
   "source": [
    "ws = Workspace.from_config()\n",
    "print(ws.name, ws.location, ws.resource_group, ws.location, sep = '\\t')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "compute_target = ws.compute_targets['gpu-nc6-v3']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "azureml.core.compute.amlcompute.AmlCompute"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "type(compute_target)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Connect to datastore"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'Input datastore tfrecords_mdv4_1 is not in the workspace; registering it...'"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<azureml.data.azure_storage_datastore.AzureBlobDatastore object at 0x11e442438>\n",
      "<azureml.data.azure_storage_datastore.AzureBlobDatastore object at 0x11e62c9e8>\n"
     ]
    }
   ],
   "source": [
    "input_datastore_name = 'tfrecords_mdv4_1'\n",
    "input_container_name = 'megadetectorv4-1'\n",
    "\n",
    "# this is actuall a miscellaneous datastore, not used for output currently\n",
    "output_datastore_name = 'artifacts'\n",
    "output_container_name = 'megadetectorv4-artifacts'\n",
    "\n",
    "input_datastore = None\n",
    "output_datastore = None\n",
    "for name, ds in ws.datastores.items():\n",
    "    if name == input_datastore_name:\n",
    "        input_datastore = ds\n",
    "    if name == output_datastore_name:\n",
    "        output_datastore = ds\n",
    "        \n",
    "if input_datastore is None:\n",
    "    'Input datastore {} is not in the workspace; registering it...'.format(input_datastore_name)\n",
    "    input_datastore = Datastore.register_azure_blob_container(workspace=ws, \n",
    "                                             datastore_name=input_datastore_name, \n",
    "                                             container_name=input_container_name,\n",
    "                                             account_name=storage_account_name, \n",
    "                                             account_key=storage_account_key,\n",
    "                                             create_if_not_exists=True)\n",
    "\n",
    "if output_datastore is None:\n",
    "    'Output datastore {} is not in the workspace; reigstering it...'.format(output_datastore_name)\n",
    "    output_datastore = Datastore.register_azure_blob_container(workspace=ws, \n",
    "                                             datastore_name=output_datastore_name, \n",
    "                                             container_name=output_container_name,\n",
    "                                             account_name=storage_account_name, \n",
    "                                             account_key=storage_account_key,\n",
    "                                             create_if_not_exists=True)\n",
    "\n",
    "print(input_datastore)\n",
    "print(output_datastore)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'megadetectorv4-1'"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "input_datastore.container_name"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "$AZUREML_DATAREFERENCE_tfrecords_mdv4_1"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "text/plain": [
       "$AZUREML_DATAREFERENCE_artifacts"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "input_data_ref = DataReference(datastore=input_datastore,\n",
    "                               data_reference_name=input_datastore_name,\n",
    "                               mode='mount')\n",
    "output_data_ref = DataReference(datastore=output_datastore,\n",
    "                               data_reference_name=output_datastore_name,\n",
    "                               mode='mount')\n",
    "\n",
    "input_data_ref\n",
    "output_data_ref"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'$AZUREML_DATAREFERENCE_artifacts'"
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "str(output_data_ref)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Support for Datasets in custom Docker images is still to come\n",
    "# input_dataset = Dataset.File.from_files(path=DataPath(datastore=input_datastore))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# input_dataset.as_named_input('tfrecords').as_mount('/tmp')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Environment setup"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "container_registry = ContainerRegistry()\n",
    "container_registry.address = 'yasiyu.azurecr.io'\n",
    "container_registry.username = 'yasiyu'\n",
    "container_registry.password = registry_pw"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "run_config = RunConfiguration()\n",
    "run_config.environment.docker.enabled = True\n",
    "run_config.environment.docker.base_image='tfodapi112:190905'\n",
    "run_config.environment.docker.base_image_registry=container_registry\n",
    "\n",
    "# GPU support: Azure automatically detects and uses the NVIDIA Docker extension when it is available.\n",
    "\n",
    "run_config.environment.python.user_managed_dependencies=True  # use your own installed packages instead of an AML created Conda env\n",
    "\n",
    "run_config.target = compute_target # specify the compute target; obscure error message: `docker image` cannot run"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Experiment\n",
    "\n",
    "Modify the parameters in the pipeline.config and add notes to tags in this section."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'/Users/siyuyang/Source/Repos/GitHub_MSFT/CameraTraps/detection/detector_training'"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "os.getcwd()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {},
   "outputs": [],
   "source": [
    "experiment_name = 'mdv4_1'\n",
    "\n",
    "exp_folder = 'experiments/megadetector_v4/200203'\n",
    "\n",
    "config_file_name = 'pipeline_baseline.config'\n",
    "\n",
    "tags = {\n",
    "    'model': 'faster_rcnn_inception_resnet_v2_atrous_coco',\n",
    "    \n",
    "    'starting_from': 'artifacts/pretrained/faster_rcnn_inception_resnet_v2_atrous_coco_2018_01_28/model.ckpt',\n",
    "    \n",
    "    'learning_rate': \"\"\"cosine_decay_learning_rate {\n",
    "          learning_rate_base: 0.0003,\n",
    "          total_steps: 104012,\n",
    "          warmup_learning_rate: 0.00003,\n",
    "          warmup_steps: 2000,\n",
    "          hold_base_rate_steps: 0\n",
    "        }\"\"\",\n",
    "    \n",
    "    'augmentations': \"\"\"baseline + horizontal flip\"\"\",\n",
    "    \n",
    "    'input_set': 'mdv4box01',\n",
    "    'train_on': 'train',\n",
    "    'val_on': 'val'\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {},
   "outputs": [],
   "source": [
    "exp = Experiment(workspace=ws, name=experiment_name)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Copy the entry script of TFODAPI to the `source_directory`, which also contains the `pipeline.config`"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'experiments/megadetector_v4/200203/model_main.py'"
      ]
     },
     "execution_count": 62,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "text/plain": [
       "'experiments/megadetector_v4/200203/pipeline_baseline.config'"
      ]
     },
     "execution_count": 62,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "copy('model_main.py', exp_folder)\n",
    "copy(os.path.join('experiments/megadetector_v4/', config_file_name), exp_folder)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Run configuration"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {},
   "outputs": [],
   "source": [
    "run_config.data_references = {\n",
    "    input_datastore_name: DataReferenceConfiguration(\n",
    "        datastore_name=input_datastore_name,\n",
    "        mode='mount'\n",
    "    ),\n",
    "    output_datastore_name: DataReferenceConfiguration(\n",
    "        datastore_name=output_datastore_name,\n",
    "        mode='mount'\n",
    "    )\n",
    "}\n",
    "\n",
    "config = ScriptRunConfig(\n",
    "    source_directory=exp_folder,\n",
    "    script='model_main.py',\n",
    "    arguments=[\n",
    "        '--model_dir', './outputs',\n",
    "        '--pipeline_config_path', config_file_name,\n",
    "        '--sample_1_of_n_eval_examples', 2  # we are sampling more val set images per eval run, but eval less often\n",
    "    ],\n",
    "    run_config=run_config\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "metadata": {},
   "outputs": [],
   "source": [
    "run = exp.submit(config, tags=tags)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'mdv4_1_1585270945_e83cf9b2'"
      ]
     },
     "execution_count": 65,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "text/plain": [
       "'Starting'"
      ]
     },
     "execution_count": 65,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "run.get_details()['runId']\n",
    "\n",
    "run.get_status()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Retrieve the run later"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "run = Run(exp, 'mdv4_trial_1580850141_379ed8f0')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'Running'"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "run.get_status()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "AML's TensorBoard requires that the events file be in .log, so not working currently."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "# The Tensorboard constructor takes an array of runs\n",
    "tb = Tensorboard([run])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "tb.LOGS_ARTIFACT_PREFIX = 'events/'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "tb.start()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "# when done, call the stop() method of the Tensorboard object, or it will stay running even after your job completes.\n",
    "tb.stop()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python [conda env:cameratraps] *",
   "language": "python",
   "name": "conda-env-cameratraps-py"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.10"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
